package extractorCode;


import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


public class link4 {
	
	private final static int INITIAL_BUF_SIZE = 65536;
	public static void main(String args[]) throws Exception
	{
	
		int count=0;
	    boolean nextIndex=true;
	
	    BufferedReader reader= new BufferedReader(new FileReader("c:\\newData2\\keywords-link4.txt"));

	    //for all keywords...
	    String keyword;
	    int lastIndex;
	    while((keyword=reader.readLine())!=null)
	    {
	    	count=0;
	    	String[] splitarr=keyword.split("\t");
	    	String key = splitarr[0];
	    	int maxPage=Integer.parseInt(splitarr[1]);
	    	System.out.println(keyword);
	       if(maxPage>-1)
	       {
	    	BufferedWriter wr = new BufferedWriter( new FileWriter("C:\\newData2\\link4-withTitle\\MediaIndonesia-"+key+".txt"));
		for(int i=0;i<=maxPage;i++)
		{	
			
		System.out.println("page"+i);	
		String page = (fetchPage("http://www.google.co.id/custom?hl=en&safe=active&client=pub-1806562855897994&cof=FORID:1%3BGL:1%3BS:http://www.mediaindonesia.com%3BL:http://www.mediaindonesia.com/images/logo_small.gif%3BLH:37%3BLW:187%3BLBGC:336699%3BLC:%230000ff%3BVLC:%23663399%3BGFNT:%230000ff%3BGIMP:%230000ff%3BDIV:%23336699%3B&domains=www.mediaindonesia.com&sitesearch=www.mediaindonesia.com&channel=3740804449&oe=ISO-8859-1&ei=lDfhTuavO-X2sQLqkOnPBg&q="+key+"&start="+(i*10)+"&sa=N"));
		System.out.println(page);
		//if(page.contains("The document has moved"));
		//{
			///System.out.println("SLEEEPPPPING!!!");
			//Thread.sleep(120000);
		
		
		//}
		page= page.replaceAll(">\\s*<", "><");
		
		page = page.replaceAll("\t", "");
		page = page.replaceAll("\b", "");
		page = page.replaceAll("\r", "");
		page = page.replaceAll("\n", "");
		page = page.replaceAll("\r\n", "");
		Matcher newsItem = Pattern.compile("<a class=\"l\"\\s*href=\"(.*?)\"", Pattern.CASE_INSENSITIVE).matcher( page );
		Matcher dateItem= Pattern.compile("<span class=\"s\">(.*?)<b>", Pattern.CASE_INSENSITIVE).matcher( page );

		Matcher titleItem= Pattern.compile("<a class=\"l\" href=\".*?\">(.*?)</a>", Pattern.CASE_INSENSITIVE).matcher( page );
	while(newsItem.find()&& dateItem.find()&&titleItem.find())
		{
	//	System.out.println(newsItem.group(1));
		
		String text = newsItem.group(1);
		String date= dateItem.group(1);
		String title = titleItem.group(1);
		nextIndex= text.contains("<span>Next</span>");
		//System.out.println(text);
		text = text.replaceAll("\\<.*?>","");
		text = text.replaceAll("[^\\p{ASCII}]", "");
		
		wr.write(text+"\t"+date+"\t"+title+"\n");
		System.out.println(text+"\t"+date+"\t"+title+"\n");
        count++;
		
		//System.out.println(text);
	//	writer.write("http://newsbusters.org"+newsItem.group(1)+"\n");
		}
		}
		System.out.println("The total number of links "+count);
		
		wr.close();
	    }}
	}
	
	private static String fetchPage(String urlString){
		StringBuilder buffer = new StringBuilder(INITIAL_BUF_SIZE);
		try {
			//URL url = new URL( urlString );
			
			
			 URL url = new URL(urlString);
             HttpURLConnection.setFollowRedirects(false);
             HttpURLConnection urlConn = (HttpURLConnection)url.openConnection();

        //    urlConn.setRequestProperty("User-agent","Mozilla/2.0.0.11");
            
          urlConn.setRequestProperty("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
            urlConn.setRequestProperty("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.3");
            urlConn.setRequestProperty("Accept","Encoding:gzip,deflate,sdch");
            urlConn.setRequestProperty("Accept","Language:en-US,en;q=0.8");
            urlConn.setRequestProperty("Cache-Control","max-age=0");
            urlConn.setRequestProperty("Connection","keep-alive");
            urlConn.setRequestProperty("Cookie","NID=51=rT0WoBSBIOyB6m9xJo9I6FHW9o6CQapffJkJ4lkaUyc8wc-TwcZSCcn45s0mbSat2KmRRC77-2-w23QD4ULju-wWpuYz61-amQqSc5XVbP0u9EaP4AykDZr9eNiI3FGO; PREF=ID=7a3bcb20c7dc0af6:U=d2fdb6c8ce18859e:FF=0:TM=1318276485:LM=1318276523:S=cKN1uTa0p4U1m2IJ");
            urlConn.setRequestProperty("Host","www.google.co.id");
             urlConn.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2");
             urlConn.connect();
             BufferedReader reader = new BufferedReader(new InputStreamReader(urlConn.getInputStream()));
             
			
			//BufferedInputStream reader = new BufferedInputStream(url.openStream());
			int temp = reader.read();
			while(temp != -1){
				buffer.append((char)temp);
				temp = reader.read();
			}
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return buffer.toString();
	}

}
